﻿#region 命名空间

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Microsoft.Practices.EnterpriseLibrary.PolicyInjection;
using WinnerCMS.Common;
using WinnerCMS.GatherEngine;
using WinnerCMS.Model;

#endregion

namespace WinnerCMS.BLL.GatherEngine
{
    /// <summary>
    ///     采集内容页
    /// </summary>
    public class GatherContentPage
    {
        #region 获取文章内容页的下页url

        /// <summary>
        ///     获取文章内容页的下页url
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="doc"></param>
        /// <param name="strWebCode"></param>
        /// <returns></returns>
        private string GetArticleNextPageURL(IEnumerable<Gather> rulesList, HtmlDocument doc, string strWebCode)
        {
            var articleNextPage = "";
            foreach (var ag in rulesList.Where(m => m.ContentPageNextUrl.Length > 0))
            {
                switch (GatherRule.Analysis(ag.ContentPageNextUrl))
                {
                    case GatherRuleType.XPath:
                        var hn = doc.DocumentNode.SelectSingleNode(ag.ContentPageNextUrl.Substring(2));
                        articleNextPage = hn == null ? string.Empty : hn.Attributes["href"].Value;
                        break;
                    case GatherRuleType.DomQuery:
                        hn = doc.SingleGet(ag.ContentPageNextUrl.Substring(2));
                        articleNextPage = hn == null ? string.Empty : hn.Attributes["href"].Value;
                        break;
                    default:
                        articleNextPage = Utilities.GetInfo(strWebCode, ag.ContentPageNextUrl);
                        break;
                }
                if (string.IsNullOrEmpty(articleNextPage) == false)
                {
                    break;
                }
            }
            if (articleNextPage.EndsWith("#") ||
                articleNextPage.StartsWith("javascript", StringComparison.CurrentCultureIgnoreCase))
            {
                return string.Empty;
            }
            return articleNextPage;
        }

        #endregion

        #region 获取文章内容页的内容

        /// <summary>
        ///     获取文章内容页的内容
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="doc"></param>
        /// <param name="strWebCode"></param>
        /// <returns></returns>
        private string GetArticleContent(IList<Gather> rulesList, HtmlDocument doc, string strWebCode)
        {
            foreach (var ag in rulesList.Where(m => m.Content.Length > 0))
            {
                string strContent;
                switch (GatherRule.Analysis(ag.Content))
                {
                    case GatherRuleType.XPath:
                        var _hn = doc.DocumentNode.SelectSingleNode(ag.Content.Substring(2));
                        strContent = _hn == null ? null : _hn.InnerHtml;
                        break;
                    case GatherRuleType.DomQuery:
                        strContent = doc.SingleGetInnerHtml(ag.Content.Substring(2));
                        break;
                    default:
                        strContent = Utilities.GetInfo(strWebCode, ag.Content);
                        break;
                }

                if (string.IsNullOrWhiteSpace(strContent) == false) //如果采集到的就跳出循环
                {
                    return strContent.Trim();
                }
            }
            return "";
        }

        #endregion

        #region 是否为忽略网址

        /// <summary>
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="url"></param>
        /// <returns></returns>
        public bool IsIgnoreUrl(IList<Gather> rulesList, string url)
        {
            foreach (var ag in rulesList.Where(m => string.IsNullOrEmpty(m.IgnoreUrl) == false))
            {
                var reg = new Regex(ag.IgnoreUrl, RegexOptions.IgnoreCase);
                if (reg.IsMatch(url))
                {
                    return true;
                }
            }
            return false;
        }

        #endregion

        #region - 变量 -

        private static readonly ArticleManage ArticleBll;

        private static readonly GatherManage GatherBll;

        private static readonly GatherHistoryManage HistoryBll;

        private IList<Gather> _rulesList;

        private Gather _gatherModel;

        #endregion

        #region - 属性 -

        public string PicPath { get; set; }

        private readonly WebParameter _wp;

        #endregion

        #region - 构造函数 -

        static GatherContentPage()
        {
            ArticleBll = PolicyInjection.Create<ArticleManage>();
            GatherBll = PolicyInjection.Create<GatherManage>();
            HistoryBll = new GatherHistoryManage();
        }

        public GatherContentPage()
        {
        }

        public GatherContentPage(WebParameter wp)
        {
            this._wp = wp;
            this._rulesList = GatherBll.GetRules(this._wp.GatherId.ToString(), this._wp.ModelId);
            this._gatherModel = GatherBll.GetInfo(int.Parse(this._wp.GatherId.ToString()));

            #region 校验基本条件

            PicPath = "";
            if (string.IsNullOrEmpty(PicPath))
            {
                PicPath = "/UpLoadFolder/";
            }

            try
            {
                var di = new DirectoryInfo(HttpServer.MapPath(PicPath));
                if (!di.Exists)
                {
                    di.Create();
                }
            }
            catch
            {
            }

            #endregion
        }

        #endregion

        #region 采集内容页

        public void GatherPageForThreading(object Url)
        {
            if (Url == null) throw new ArgumentNullException("Url");

            var url = new Uri(Url.ToString());
            if (this._rulesList == null)
                this._rulesList = GatherBll.GetRules(this._wp.GatherId.ToString(), this._wp.ModelId);
            if (this._gatherModel == null)
                this._gatherModel = GatherBll.GetInfo(int.Parse(this._wp.GatherId.ToString()));

            GatherPage(url, false);
        }

        /// <summary>
        ///     采集内容页
        /// </summary>
        /// <param name="Url"></param>
        /// <returns></returns>
        public ContentPageGatherResult GatherPage(object Url)
        {
            try
            {
                var url = new Uri(Url.ToString());
                return GatherPage(url, false);
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex);

                var cpgr = new ContentPageGatherResult
                {
                    URL = Url.ToString(),
                    Messages = ex.Message,
                    Gathered = false,
                    Type = ContentPageGatherResultType.Alert
                };

                return cpgr;
            }
        }

        /// <summary>
        ///     采集内容页
        /// </summary>
        /// <param name="url"></param>
        /// <param name="fullArticleScaned">已经扫描过全文文章地址</param>
        private ContentPageGatherResult GatherPage(Uri url, bool fullArticleScaned)
        {
            //LogHelper.Verbose("this._wp.GatherId.ToString()：" + gather_model.ID + "\r\nUrl：" + _Url.ToString());

            var cpgr = new ContentPageGatherResult
            {
                URL = url.ToString(),
                Messages = "忽略采集的网址。",
                Gathered = false,
                Type = ContentPageGatherResultType.Right
            };
            if (IsIgnoreUrl(this._rulesList, url.ToString()))
            {
                return cpgr;
            }
            var gathered = false;

            cpgr = new ContentPageGatherResult();

            try
            {
                var articleShowUrl = url.ToString();
                cpgr.URL = articleShowUrl;


                var downImg = this._gatherModel.CanDownImg;
                var drawWaterMark = this._gatherModel.CanDrawWaterMark;

                bool urlGathered;
                lock (GatherBll)
                {
                    urlGathered = HistoryBll.Exists(articleShowUrl);
                }

                if (this._wp.IsUpdate || urlGathered == false)
                {
                    #region 采集文章

                    IList<string> contentList = new List<string>();
                    var model = new Article
                    {
                        CategoryId = this._gatherModel.CategoryId,
                        Tags = this._gatherModel.Category.CategoryName,
                        Editor = this._wp.AdminName ?? UserManage.Current.UserName
                    };

                    var engine = new NetHelper
                    {
                        EncodingString = this._rulesList.First(x => x.PID == 0).Encoding
                    };
                    var info = engine.Get(url);

                    if (info.StatusCode == HttpStatusCode.NotFound)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "访问远程资源发生一般性错误。" + info.Code;
                        goto Exit;
                    }
                    if (info.StatusCode != HttpStatusCode.OK)
                    {
                        cpgr.Type = ContentPageGatherResultType.Err;
                        cpgr.Messages = "访问远程资源错误。" + info.Code;
                        goto Exit;
                    }
                    if (info.HTML.Length < 2048)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集到网页过小。";
                        goto Exit;
                    }
                    cpgr.Charset = info.Charset;

                    var strWebCode = info.HTML;
                    var doc = new HtmlDocument();
                    //载入HTML
                    doc.LoadHtml(strWebCode);

                    #region 获取内容标题

                    foreach (var ag in this._rulesList.Where(m => m.ContentTitle.Length > 0))
                    {
                        switch (GatherRule.Analysis(ag.ContentTitle))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.ContentTitle.Substring(2));
                                model.ContentTitle = hn == null ? null : hn.InnerText.Trim();
                                break;
                            case GatherRuleType.DomQuery:
                                model.ContentTitle = doc.SingleGetInnerText(ag.ContentTitle.Substring(2));
                                break;
                            default:
                                model.ContentTitle = Utilities.GetInfo(strWebCode, ag.ContentTitle);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.ContentTitle) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.ContentTitle))
                    {
                        cpgr.Type = ContentPageGatherResultType.Err;
                        cpgr.Messages = "采集不到标题。";
                        goto Exit;
                    }

                    #endregion

                    int existsArticleId; //已经存在的文章的ID
                    lock (GatherBll)
                    {
                        if (ArticleBll.Exists(model.ContentTitle, out existsArticleId)) //查找相同的文章
                        {
                            HistoryBll.LogUrl(articleShowUrl);

                            if (this._wp.IsUpdate == false)
                            {
                                cpgr.Type = ContentPageGatherResultType.Right;
                                cpgr.Messages = "文章已经存在。";
                                goto Exit;
                            }
                        }
                    }

                    #region 获取全文地址

                    if (fullArticleScaned == false)
                    {
                        foreach (var ag in this._rulesList.Where(m => string.IsNullOrEmpty(m.FullArticleUrl) == false))
                        {
                            string fullArticleUrl;
                            switch (GatherRule.Analysis(ag.FullArticleUrl))
                            {
                                case GatherRuleType.XPath:
                                    var hn = doc.DocumentNode.SelectSingleNode(ag.FullArticleUrl.Substring(2));
                                    fullArticleUrl = hn == null ? null : hn.InnerText;
                                    break;
                                case GatherRuleType.DomQuery:
                                    hn = doc.SingleGet(ag.FullArticleUrl.Substring(2));
                                    fullArticleUrl = hn == null ? string.Empty : hn.Attributes["href"].Value;
                                    break;
                                default:
                                    fullArticleUrl = Utilities.GetInfo(strWebCode, ag.FullArticleUrl);
                                    break;
                            }

                            if (string.IsNullOrEmpty(fullArticleUrl) == false)
                            {
                                fullArticleUrl = Utilities.FormatUrl(articleShowUrl, fullArticleUrl);
                                return GatherPage(new Uri(fullArticleUrl), true);
                            }
                        }
                    }

                    #endregion

                    var isScan = false;
                    //文章来源
                    foreach (var ag in this._rulesList.Where(m => m.ComeFrom.Length > 0))
                    {
                        isScan = true;
                        switch (GatherRule.Analysis(ag.ComeFrom))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.ComeFrom.Substring(2));
                                model.ComeFrom = hn == null ? null : hn.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                model.ComeFrom = doc.SingleGetInnerText(ag.ComeFrom.Substring(2));
                                break;
                            default:
                                model.ComeFrom = Utilities.GetInfo(strWebCode, ag.ComeFrom);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.ComeFrom) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.ComeFrom) && isScan)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章来源。";
                    }
                    isScan = false;
                    //文章作者
                    foreach (var ag in this._rulesList.Where(m => m.Author.Length > 0))
                    {
                        isScan = true;
                        switch (GatherRule.Analysis(ag.Author))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.Author.Substring(2));
                                model.Author = hn == null ? null : hn.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                model.Author = doc.SingleGetInnerText(ag.Author.Substring(2));
                                break;
                            default:
                                model.Author = Utilities.GetInfo(strWebCode, ag.Author);
                                break;
                        }

                        if (string.IsNullOrEmpty(model.Author) == false)
                        {
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(model.Author) && isScan)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章作者。";
                        isScan = false;
                    }

                    #region 采集文章更新时间

                    foreach (var ag in this._rulesList.Where(m => m.UpdateTime.Length > 0))
                    {
                        string strdatatime;
                        switch (GatherRule.Analysis(ag.UpdateTime))
                        {
                            case GatherRuleType.XPath:
                                var hn = doc.DocumentNode.SelectSingleNode(ag.UpdateTime.Substring(2));
                                strdatatime = hn == null ? null : hn.InnerText;
                                break;
                            case GatherRuleType.DomQuery:
                                strdatatime = doc.SingleGetInnerText(ag.UpdateTime.Substring(2));
                                break;
                            default:
                                strdatatime = Utilities.GetInfo(strWebCode, ag.UpdateTime);
                                break;
                        }

                        if (string.IsNullOrEmpty(strdatatime))
                        {
                            continue; //采集不到就继续下一个规则
                        }
                        strdatatime = strdatatime.Trim();
                        DateTime dt;
                        if (DateTime.TryParse(strdatatime, out dt))
                        {
                            var s = dt.ToString("HH:mm:ss.ffff");
                            if (s == "00:00:00.0000")
                            {
                                var e = DateTime.Now.ToString(" HH:mm:ss.ffff");
                                dt = DateTime.Parse(string.Concat(dt.ToString("yyyy-MM-dd"), e));
                            }
                            s = dt.ToString("ss.ffff");
                            if (s == "00.0000")
                            {
                                var e = DateTime.Now.ToString("ss.ffff");
                                dt = DateTime.Parse(string.Concat(dt.ToString("yyyy-MM-dd HH:mm:"), e));
                            }
                            model.UpdateTime = dt;
                            break;
                        }
                    }
                    if (model.UpdateTime == DateTime.MinValue)
                    {
                        cpgr.Type = ContentPageGatherResultType.Alert;
                        cpgr.Messages = "采集不到文章更新时间。";
                    }

                    #endregion

                    var strContent = GetArticleContent(this._rulesList, doc, strWebCode);

                    if (string.IsNullOrEmpty(strContent))
                    {
                        cpgr.Type = ContentPageGatherResultType.Err;
                        cpgr.Messages = "文章内容采集不到。";
                        goto Exit;
                    }


                    contentList.Add(strContent);

                    //采集文章页内的下页
                    if (this._gatherModel.ContentPageNextUrl.Length > 0 && fullArticleScaned == false)
                    {
                        //文章页下一页
                        var articleNextPage = GetArticleNextPageURL(this._rulesList, doc, strWebCode);

                        if (articleNextPage.Length > 0 &&
                            !Utilities.FormatUrl(articleShowUrl, articleNextPage)
                                      .Equals(url.ToString(), StringComparison.CurrentCultureIgnoreCase))
                        {
                            articleNextPage = Utilities.FormatUrl(articleShowUrl, articleNextPage);
                            do
                            {
                                info = engine.Get(articleNextPage, articleShowUrl);

                                if (string.IsNullOrEmpty(info.Code) == false)
                                {
                                    cpgr.Type = ContentPageGatherResultType.Err;
                                    cpgr.Messages = "访问“下一页”出错。" + info.Code;
                                    goto Exit;
                                }
                                strWebCode = info.HTML;

                                //载入HTML
                                doc = new HtmlDocument();
                                doc.LoadHtml(info.HTML);

                                strContent = GetArticleContent(this._rulesList, doc, strWebCode);

                                if (string.IsNullOrEmpty(strContent)) //如果都循环完了还是采集不到
                                {
                                    cpgr.Type = ContentPageGatherResultType.Alert;
                                    cpgr.Messages = "采集不到“下一页”(" + articleNextPage + ")文章内容。";
                                }

                                #region 检测是否死循环

                                if (contentList.Contains(strContent))
                                {
                                    cpgr.Type = ContentPageGatherResultType.Err;
                                    cpgr.Messages = "出现疑似死循环，请检查采集规则。";
                                    goto Exit;
                                }

                                #endregion

                                contentList.Add(strContent);

                                var tempArticleNextPage = GetArticleNextPageURL(this._rulesList, doc, strWebCode);
                                if (string.IsNullOrWhiteSpace(tempArticleNextPage))
                                {
                                    break;
                                }

                                tempArticleNextPage = Utilities.FormatUrl(articleShowUrl, tempArticleNextPage);

                                if (articleNextPage.Equals(url.ToString(), StringComparison.CurrentCultureIgnoreCase)
                                    ||
                                    articleNextPage.Equals(tempArticleNextPage,
                                        StringComparison.CurrentCultureIgnoreCase)
                                    ) //防止最后一页的“下一页“就是本身
                                {
                                    articleNextPage = string.Empty;
                                }
                                else
                                {
                                    articleNextPage = tempArticleNextPage;
                                }
                            } while (articleNextPage.Length > 0);
                        }
                    }

                    //过滤
                    model.Content = string.Join("@Next@", contentList.ToArray());
                    model.Content = Filter.GatherFilter(model.Content, this._gatherModel.Filter,
                        this._gatherModel.Replace);
                    model.Content = Utilities.FormatUrlInHtml(articleShowUrl, model.Content, this._wp.IsLeech);
                    model.Content = Filter.FilterUrl(model.Content); //去除多余的网页中的链接
                    model.Content = HTML.NoHtml(model.Content,
                        WebSourceTag.Note | WebSourceTag.Invisible | WebSourceTag.Script | WebSourceTag.Style);


                    var yes = true; //是否采集文章。


                    var picurLs = Utilities.Get_PicUrl(model.Content, url.ToString()).ToList();
                    if (picurLs.Count > 0 && this._wp.IsTest == false)
                    {
                        //如果要下载图片而且要采集有图片的文章
                        if (downImg)
                        {
                            if (this._wp.IsTest == false && downImg)
                            {
                                //是允许采集
                                var allowDownPic = engine.UrlExist(picurLs, this._wp.BadLinkNum);

                                #region 采集图片和加水印

                                if (allowDownPic)
                                {
                                    var myWebClient = new WebClient();
                                    var drawingImage = new DrawingWaterMarkImage();
                                    foreach (var getPicUrl in picurLs)
                                    {
                                        var name = HttpClient.UrlFileName(getPicUrl);

                                        //下载图片
                                        try
                                        {
                                            myWebClient.DownloadFile(getPicUrl,
                                                HttpServer.MapPath(PicPath + "\\" + name));
                                        }
                                        catch (Exception ex)
                                        {
                                            LogHelper.Error(ex);
                                            yes = false;
                                            cpgr.Type = ContentPageGatherResultType.Err;
                                            cpgr.Messages = (ex.ToString());
                                        }
                                        finally //这个图片如果采集成功就替换成本地链接
                                        {
                                            if (drawWaterMark && yes)
                                            {
                                                drawingImage.WatherMark(HttpServer.MapPath(PicPath + "\\" + name));
                                                //加盖水印
                                            }
                                            model.Content = model.Content.Replace(getPicUrl, PicPath + name);
                                        }
                                    }
                                }
                                else
                                {
                                    cpgr.Messages = ("文章含有图片死链接，丢弃它。");
                                    yes = false;
                                }

                                #endregion
                            }
                        }
                        else
                        {
                            yes = true;
                        }
                    }

                    if (yes && !this._wp.IsTest)
                    {
                        Result res;

                        #region 使用默认缩略图

                        if (picurLs.Count > 0)
                        {
                            model.IncludePic = true;
                            model.PicUrl = picurLs[0];
                        }

                        #endregion

                        if (this._wp.IsUpdate == false)
                        {
                            #region 添加文章

                            model.Id = 0;
                            model.Subtitle = string.Empty;
                            model.PicUrl = string.Empty;
                            model.Stars = 3;
                            model.PicUrl = string.Empty;
                            model.AllowComment = true;
                            model.IsDeleted = true;
                            model.IsChecked = true;

                            res = ArticleBll.Save(model, this._wp.UserId);

                            #endregion
                        }
                        else
                        {
                            #region 修改文章

                            if (existsArticleId == 0) goto Exit;

                            model.Id = existsArticleId;
                            if (model.UpdateTime == DateTime.MinValue)
                            {
                                model.UpdateTime = DateTime.Now;
                            }
                            res = ArticleBll.EditForGather(model);

                            #endregion
                        }
                        if (string.IsNullOrEmpty(res.Errors))
                        {
                            HistoryBll.LogUrl(articleShowUrl);
                            cpgr.Messages = ("文章采集成功");
                            cpgr.Type = ContentPageGatherResultType.Right;
                            gathered = true;
                        }
                        else
                        {
                            cpgr.Type = ContentPageGatherResultType.Err;
                            cpgr.Messages = res.Errors;
                        }
                    }
                    else
                    {
                        if (this._wp.IsTest)
                        {
                            gathered = true;
                            cpgr.Messages = "文章采集成功。(测试采集未入库)";
                        }
                        else
                        {
                            cpgr.Messages = "自动跳过采集。";
                        }
                    }

                    #endregion
                }
                else
                {
                    cpgr.Messages = "已经采集过。";
                    gathered = false;
                }
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex);
                cpgr.Type = ContentPageGatherResultType.Err;
                cpgr.Messages = ex.Message;
            }
            Exit:
            if (gathered)
            {
                GatherBll.LastWork(int.Parse(this._wp.GatherId.ToString()));
            }
            cpgr.Gathered = gathered;
            cpgr.Messages += cpgr.Charset;

            return cpgr;
        }

        #endregion
    }
}